from urllib import request, parse
from lxml import etree
import ssl
import time
import threading
import pymysql
import json
import re

# 取消 https 代理验证
ssl._create_default_https_context = ssl._create_unverified_context


# 关闭光标对象
# cursor.close()
# # 关闭数据库连接
# conn.close()


class gateSpider:
    def __init__(self):
        self.list_dict = self.dealJson()
        # self.list_dict = ['一', '礵', '嶒']
        self.num = 0

        # 爬取地址
        self.https = "https://hanyu.baidu.com"
        self.host = "https://hanyu.baidu.com/s?ptype=zici&wd="

    def dealJson(self):
        text_list = []
        with open("fan.json", "r") as f:
            load_dict = json.load(f)
            for item in load_dict['text']:
                text_list.append(item)
            return text_list
        # with open('1.txt', 'a+') as f:
        #     f.write(str(result_dict) + "\n")

    def loadPage(self, num):
        # 报头
        user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Safari/537.36'
        headers = {'User-Agent': user_agent}
        url = self.host + parse.quote(self.list_dict[num])
        req = request.Request(url, headers=headers)
        try:
            response = request.urlopen(req)
            # 获取每页的HTML源码字符串
            html = response.read().decode('utf-8')
            # 解析html 为 HTML 文档
            selector = etree.HTML(html)
            # 调用dealPage() 处理杂七杂八
            self.dealPage(selector, num)
        except Exception:
            print(url, Exception)

    def dealPage(self, selector, num):
        result_dict = {
            'is_delete': 0,  # 是否删除
            'name': self.list_dict[num],  # 名称
            'is_simplified': 0,  # 简体
            'is_traditional': 1,  # 繁体
            'ralation_id': None,  # 关联ID
        }

        # 图片
        try:
            image_url = selector.xpath("//div[@id='header-img']//div[@class='alter-text']/@style")[0],
            result_dict['image_url'] = re.findall(r'[(](.*?)[)]', str(list(image_url)))[0]
        except IndexError:
            result_dict['image_url'] = selector.xpath("//img[@id='word_bishun']/@data-gif")[0],
        except Exception:
            result_dict['image_url'] = None

        # 拼音
        try:
            result_dict['spell'] = selector.xpath("//div[@id='pinyin']//b/text()")[0],
        except Exception:
            result_dict['spell'] = None

        # 部首
        try:
            result_dict['radicals'] = selector.xpath("//li[@id='radical']//span/text()")[0],
        except Exception:
            result_dict['radicals'] = None

        # 笔画
        try:
            result_dict['stroke_count'] = selector.xpath("//li[@id='stroke_count']//span/text()")[0],
        except Exception:
            result_dict['stroke_count'] = None

        # 五行
        try:
            result_dict['five_elements'] = selector.xpath("//li[@id='wuxing']//span/text()")[0],
        except Exception:
            result_dict['five_elements'] = None

        # 五笔
        try:
            result_dict['wubi'] = selector.xpath("//li[@id='wubi']//span/text()")[0],
        except Exception:
            result_dict['wubi'] = None

        # 基本释义
        try:
            base_paraphrase = selector.xpath("//div[@id='basicmean-wrapper']//dl//dd")[0]
            # 转为字符串
            base_paraphrase = etree.tostring(base_paraphrase, method='html', encoding="utf-8").decode('utf-8').replace(
                ' ', '').replace('\n', '')
            result_dict['base_paraphrase'] = base_paraphrase
        except Exception:
            result_dict['base_paraphrase'] = None

        # 详细释义
        try:
            detail_paraphrase = selector.xpath("//div[@id='detailmean-wrapper']//dl//dd")[0]
            # 转为字符串
            detail_paraphrase = etree.tostring(detail_paraphrase, method='html', encoding="utf-8").decode(
                'utf-8').replace(
                ' ', '').replace('\n', '')
            result_dict['detail_paraphrase'] = detail_paraphrase
        except Exception:
            result_dict['detail_paraphrase'] = None

        # 相关词组
        try:
            relate_word = selector.xpath(
                "//div[@id='zuci-wrapper']//div[@class='tab-content']/a[not(contains(@style,'display: none;'))]")
            relate_word_list = []
            for item in relate_word:
                relate_word_list.append(item.xpath('text()')[0])
            relate_word_list.pop()
            result_dict['relate_word'] = relate_word_list
        except Exception:
            result_dict['relate_word'] = None

        print(result_dict)

        '''

        # 基本释义 这个是处理过的
        base_paraphrase = selector.xpath("//div[@id='basicmean-wrapper']//dl//dd/p")
        base_paraphrase_list = []
        for item in base_paraphrase:
            try:
                base_paraphrase_list.append(item.xpath('text()')[0] + item.xpath('span/text()')[0])
            except Exception:
                base_paraphrase_list.append("")

        result_dict['base_paraphrase'] = base_paraphrase_list

        # 近义词
        synonym = selector.xpath("//div[@id='synonym']//a")
        synonym_list = []
        for item in synonym:
            synonym_list.append(item.xpath('text()')[0])
        result_dict['synonym'] = synonym_list

        # 反义词
        antonym = selector.xpath("//div[@id='antonym']//a")
        antonym_list = []
        for item in antonym:
            antonym_list.append(item.xpath('text()')[0])
        result_dict['antonym'] = antonym_list

        # 相关谜语
        riddle = selector.xpath("//p[@id='riddle']")
        riddle_list = []
        for item in riddle:
            riddle_list.append(item.xpath('text()')[0])
        result_dict['riddle'] = riddle_list

        # 相近字
        xiangjinzi = selector.xpath("//div[@class='recmd-wrapper'][1]//ul/li")
        xiangjinzi_list = []
        for item in xiangjinzi:
            xiangjinzi_list.append({'img': item.xpath('a[1]/img/@data-src')[0], 'text': item.xpath('a[2]/text()')[0]})
        result_dict['xiangjinzi'] = xiangjinzi_list

        # 热搜字
        resouzi = selector.xpath("//div[@class='recmd-wrapper'][2]//ul/li")
        resouzi_list = []
        for item in resouzi:
            resouzi_list.append({'img': item.xpath('a[1]/img/@data-src')[0], 'text': item.xpath('a[2]/text()')[0]})
        result_dict['resouzi'] = resouzi_list

        # 百科释义
        result_dict['baike_wrapper'] = selector.xpath(
            "normalize-space(//div[@id='baike-wrapper']//div[@class='tab-content']//p/text())")

        # 英文翻译
        result_dict['fanyi-wrapper'] = selector.xpath("normalize-space(//div[@id='fanyi-wrapper']//dl/dt/text())")
        
        '''

        self.sendPage(result_dict)

    def sendPage(self, result_dict):

        # with open('1.txt', 'a+') as f:
        #     f.write(str(result_dict) + "\n")

        # 连接database
        conn = pymysql.connect(host='localhost', user='root', password='wjt502502', database='test', charset='utf8')
        # 得到一个可以执行SQL语句的光标对象
        cursor = conn.cursor()
        sql = """
            INSERT INTO dictionary(
            is_delete,
            name,
            spell,
            radicals,
            stroke_count,
            five_elements,
            wubi,
            base_paraphrase,
            relate_word,
            image_url,
            is_simplified,
            is_traditional,
            detail_paraphrase,
            ralation_id)
            VALUES(
            %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s
        );
        """
        is_delete = result_dict['is_delete']
        name = result_dict['name']
        spell = result_dict['spell'],
        radicals = result_dict['radicals'],
        stroke_count = result_dict['stroke_count'],
        five_elements = result_dict['five_elements'],
        wubi = result_dict['wubi'],
        base_paraphrase = result_dict['base_paraphrase'],
        relate_word = str(result_dict['relate_word']),
        image_url = result_dict['image_url'],
        is_simplified = result_dict['is_simplified'],
        is_traditional = result_dict['is_traditional'],
        detail_paraphrase = result_dict['detail_paraphrase'],
        ralation_id = result_dict['ralation_id'],
        # SQL 插入语句
        try:
            # 执行sql语句
            cursor.execute(sql, [is_delete, name, spell, radicals, stroke_count, five_elements, wubi, base_paraphrase,
                                 relate_word, image_url, is_simplified, is_traditional, detail_paraphrase, ralation_id])
            # 提交到数据库执行
            conn.commit()
        except:
            # 如果发生错误则回滚
            conn.rollback()
        # 关闭数据库连接
        conn.close()

    def startWork(self):
        # t = Timer(900, self.startWork)
        # t.start()  # 定时执行
        # self.loadPage()
        pollStart = threading.Thread(target=self.poll)
        pollStart.start()

    def poll(self):
        while True:
            try:
                self.loadPage(self.num)
                self.num += 1
            except Exception:
                pass
            # time.sleep(5)


if __name__ == '__main__':
    # 实例化 Spider 对象
    gateSpider().startWork()
